package sky.demo.ml.word2vector;

import java.io.BufferedReader;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStreamWriter;

import org.deeplearning4j.text.tokenization.tokenizer.DefaultStreamTokenizer;
import org.deeplearning4j.text.tokenization.tokenizer.TokenPreProcess;
import org.deeplearning4j.text.tokenization.tokenizer.Tokenizer;
import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory;
import org.fnlp.nlp.cn.tag.CWSTagger;
import org.fnlp.util.exception.LoadModelException;

/**
 * 
 * @author zhoujinhuang
 *
 */
public class FNLPTokenizerFactory implements TokenizerFactory {

	private TokenPreProcess tokenPreProcess;
	private CWSTagger tag;

	public FNLPTokenizerFactory(String modelFile) throws LoadModelException {
		setModelFile(modelFile);
	}

	@Override
	public Tokenizer create(String toTokenize) {
		FNLPTokenizer t = new FNLPTokenizer(tag, toTokenize);
		t.setTokenPreProcessor(tokenPreProcess);
		return t;
	}

	@Override
	public Tokenizer create(InputStream toTokenize) {
		Tokenizer t = null;
		try {
			t = new FNLPTokenizer(tag, toTokenize);
			t.setTokenPreProcessor(tokenPreProcess);
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return t;
	}

	@Override
	public void setTokenPreProcessor(TokenPreProcess preProcessor) {
		this.tokenPreProcess = preProcessor;
	}

	public void setModelFile(String file) throws LoadModelException {
		tag = new CWSTagger(file);
	}

	@Override
	public TokenPreProcess getTokenPreProcessor() {
		return tokenPreProcess;
	}

	public static void main(String[] args) throws LoadModelException, IOException {
		String outFile = "out.txt";
		String filePath = "text/src.txt";
		// String content = FileUtils.readFileToString(new File(filePath),
		// "UTF-8");

		String modelFile = "D:/third_code/fnlp/models/seg.m";
		FNLPTokenizerFactory factory = new FNLPTokenizerFactory(modelFile);

		// System.out.println(fnlpPreprocessor.preProcess(content));
		BufferedReader reader = new BufferedReader(new FileReader(filePath));
		FileOutputStream ofs = new FileOutputStream(outFile);
		OutputStreamWriter writer = new OutputStreamWriter(ofs, "UTF-8");
		String line = reader.readLine();
		System.out.println(line);
		while (line != null) {
			Tokenizer create = factory.create(line);
			while (create.hasMoreTokens()) {
				String nextToken = create.nextToken();
				writer.write(nextToken);
				writer.write(' ');
				System.out.print(nextToken);
				System.out.print(' ');
			}
			writer.write('\n');
			line = reader.readLine();
			System.out.println(line);
		}
		writer.flush();
		writer.close();
		System.out.println("end ..");

	}
}
